"""
获取json
json变成pd( dicTojson)
pd加工(pandasTore)
"""
import code
import os
import sys

# sys.path.append(os.path.dirname(os.path.dirname(__file__)))
import numpy as np
import functools
sys.path.append(os.path.dirname(__file__))
# print(sys.path)
import requests
import pandas as pd
import json
import time
import random
import dicTojson
import target_url as t_u
import com
from tusharecopy.stock import cons as ct
import simplejson
from pandas.io.json import json_normalize

requests.packages.urllib3.disable_warnings()
import  tusharecopy as tscopy
#1.
class Request_url:
    """网址获取，变成True 爬取网站成功-> {"globalId":"786e4c21-70dc-435a-93bb-39","message":"OK","status":0,"code":0,"data":[{"calc"""
    def __init__(self,url,headers):
        self.url=url
        self.headers = headers
    def get(url:str,headers:str)->dict:
        r = requests.get(url=url, headers=headers,verify=False)#增加了verify=False参数
        if r.status_code==200:
            return True,"爬取网站成功",r.text
        else:
            return False,"爬取网站失败",url
    def post(url, headers,params)->dict:
        r = requests.post(url=url, headers=headers,json=params)
        if r.status_code == 200:
            return True, "爬取网站成功", r.text
        else:
            return False, "爬取网站失败", url
def choose(code:str)->dict:
    """通过Request_url，将dict。json 变成pd,并保存"""
    if code=="同花顺top100":
        # 获取数据
        flag, msg, ths_dict = Request_url.get(t_u.top_100["同花顺"]["url"], t_u.top_100["同花顺"]["headers"])

        #print(flag, msg, ths)
        #json变成pd,pd加工
        dicTojson.ths(ths_dict)
    if code == "东方财富top100":
        flag,msg,dfcf_dict=Request_url.post(t_u.top_100["东方财富"]["url"],t_u.top_100["东方财富"]["headers"],t_u.top_100["东方财富"]["params"])
        #print(flag,msg,dfcf)
        dicTojson.dfcf(dfcf_dict)
#2.
class Dfcf_rank:
    def __init__(self,code,time_var):
        self.code=code
        self.time_var=time_var
        self.d_dict=com.dfcf_shszcode(self.code,time_var)
        self.url=self.d_dict["{}".format(time_var)]["url"]
        self.headers=self.d_dict["{}".format(time_var)]["headers"]
        self.params= self.d_dict["{}".format(time_var)]["params"]
    @property
    def hot_rank(self):
        flag, msg, dfcf = Request_url.post(self.url, self.headers,self.params)
        print(flag, msg, dfcf)
        result_pd = pd.DataFrame(json.loads(dfcf)['data'])
        if not result_pd.empty:
            if self.time_var=="now实时":
                return result_pd['rank'].tolist()[-1]
            if self.time_var == "his历史":
                return result_pd['rank'].tolist()
        if result_pd.empty:
            print("网站人气暂时没有数据")
            return None
"""class Dfcf_rank.hot_rank 面条版
def hot_rank(code):
    dfcf_dict = com.dfcf_shszcode(code)
    flag, msg, dfcf = Request_url.post(dfcf_dict["now实时"]["url"], dfcf_dict["now实时"]["headers"],dfcf_dict["now实时"]["params"])
    print(flag, msg, dfcf)
    result_pd = pd.DataFrame(json.loads(dfcf)['data'])
    if not result_pd.empty:
        "137  2021-09-03 22:50:00   415
138  2021-09-03 23:00:00   415
139  2021-09-03 23:10:00   418"
        #print(result_pd['rank'].tolist()[-1])
        return result_pd['rank'].tolist()[-1]
    if result_pd.empty:
        print("网站人气暂时没有数据")
        return None"""



def zhangtingfor30days():
    time_list = com.time_saveandget(choose="day_list", day=5)
    df = pd.DataFrame()
    zt=t_u.dxw["涨停"]
    for day_time in time_list:
        url = '{}{}{}'.format(zt["url"], day_time,zt["url1"])
        print(url)
        flag, msg, dxw_dic = Request_url.get(url,headers='')
        print(flag, msg, dxw_dic)
        time.sleep(random.randint(130,150))
        new= dicTojson.zhangting.main(dxw_dic)  # 变成df
        df = pd.concat([df, new])
    dicTojson.dxw_path_save(df, i_day=time_list[-1], path="ztfor30")  # 保存


    # json_list= json.loads(ths_dic)["answer"]["components"][0]["data"]["datas"]
    # df_=pd.DataFrame(json_list)
    # return df_


class dxw:
    """"
    涨停(只能是近15天)、龙虎榜(从2015年起)
    dxw.daban(name="zhangting",day="")/dxw.daban(name="zhangting",day="20210930")
    dxw.daban(name="longhu",day="")/dxw.daban(name="longhu",day="20210930")
    个股
    dxw.gegu(day="")/_gegu_except(day="{}".format(time_day)
    异动
    dxw.yidong()


    """
    global today_time
    today_time = com.time_saveandget()

    def __init__(self):
        pass
    def daban(name,day,*args,**kwargs):#dxw("涨停","龙虎榜")/dxw("龙虎榜")
        """

        :param name: zhangting/longhu
        :param args:
        :param kwargs: today_time="20210929"
        :return:df
        """
        if day == "":
            day= com.time_saveandget()
        # print('->',name,day)
        url=t_u.dxw["{}".format(name)].format(day)
        #print(name,url)zhangting_header
        flag, msg, dxw_dic = Request_url.get(url, headers=t_u.dxw["zhangting_header"])
        #print(flag, msg, dxw_dic)
        if flag:
            df = dicTojson.zhangting.main(dxw_dic)  # 变成df

            dicTojson.dxw_path_save(df, i_day=day, path=name)  # 保存
            print("->成功保存数据dxw/zhangting/{}.json".format(day))
            return df
        else :
            print("爬取失败，请检查flag,msg,dxw_dict")

    #————————————————————————————————————————————————————————————
    def dxw_gegutosave(func):
        @functools.wraps(func)
        def wrapper(*args, **kwargs):
            res = func(*args, **kwargs)
            # print(res)
            path = os.path.dirname(os.path.dirname(__file__))
            if kwargs["day"]=="":
                time_ = com.time_saveandget()
            else:
                time_=kwargs["day"]
            path = os.path.join(path, "db", "dxw", "gegu", '{}.json'.format(time_))
            print("开始保存到dxw/gegu/{}中".format(time_))
            res.to_csv("{}".format(path), encoding="utf-8")
        return wrapper
    def gn_bkcode_b1(*args, **kwargs):
        # try与except 是针对gn_bkcode所打的补丁#
        try:
            global gncode
            gncode = kwargs["gncode"]
        except:
            pass
    def geguwrapper(func):
        @functools.wraps(func)
        def wrapper(*args, **kwargs):
            df_return = pd.DataFrame()
            print("->回测时间:",kwargs["day"])
            for i in range(1, 30):

                #try与except 是针对gn_bkcode所打的补丁#
                dxw.gn_bkcode_b1(*args, **kwargs)
                time_sleep_random = random.uniform(1.5, 3)
                if time_sleep_random > 2.5:
                    time.sleep(5)
                time.sleep(time_sleep_random)
                # if i%20==0:
                #     time.sleep(180)
                # print("page", i, kwargs, time_sleep_random)
                df = func(day=kwargs["day"], page=i)
                if df.empty == True:
                    # df_return=chajian.dftore(df_return)
                    return df_return
                #if 是 gn_bkcode补丁
                if len(df)<19:
                    print("->完成板块gncode中股票爬取")
                    return df_return
                df_return = df_return.append(df, ignore_index=True)
            return df_return

        return wrapper

    global new_col_dic
    new_col_dic={"col0":['股票', '涨幅',"主动净买","主力资金","板块","现价","股性","主动占比","主力占比","涨速","换手","量比","成交额","流通值","市盈(动)","3日主动","10日主动"],
    "col1":['股票', '涨幅',"3日涨幅","10日涨幅","板块","现价","股性","主动净买","主力资金","主动占比","主力占比","涨速","换手","量比","成交额","流通值","市盈(动)"]}
    """
    @dxw_gegutosave
    @geguwrapper
    def _gegu(day='20210528',page="1",new_col_=new_col_dic["col1"]):
        ct._write_console()
        # 个股涨幅榜
        if day=='20210528':
            day==""#今日
        else:
            day==day
        url =t_u.dxw["个股_涨幅榜_涨幅"].format(day,page)
        flag, msg, dxw_dic = Request_url.get(url, headers="")
        if flag:
            r_list = simplejson.loads(dxw_dic, strict=False)["data"]["stock_list"]  # stock_list
            df_ = pd.DataFrame()
            for keys, values in pd.DataFrame(r_list['list']).items():
                ##print(keys,values)
                text_ = pd.json_normalize(values).text  # code
                df_ = pd.concat([df_, text_], axis=1)
                if keys == 0:  # 机构与游资
                    list_ = []
                    for i in range(len(pd.DataFrame(r_list['list']))):
                        try:
                            type_num = pd.DataFrame(r_list['list'])[0][i]["icon_target"]["param"]["type"]  # 返回3为游资，返回6为机构
                            if type_num == 6:
                                # print("机构")
                                list_.append("机构")
                            if type_num == 3:
                                # print("游资")
                                list_.append("游资")

                        except:
                            # print("NAN")
                            list_.append("NAN")
                if keys == 4:  # 板块
                    # print("第4行",keys)
                    list_bankuai_code = pd.json_normalize(values)["param.code"]
                    try:
                        list_bankuai_remen = pd.json_normalize(values)["attr_tag"]
                    except:
                        list_bankuai_remen = []
            # 换列名字
            new_col = new_col_  # ['股票', '涨幅',"3日涨幅","10日涨幅","板块","现价","股性","主动净买","主力资金","主动占比","主力占比","涨速","换手","量比","成交额","流通值","市盈(动)"]
            df_.columns = new_col
            df_["机构"] = list_
            df_["板块code"] = list_bankuai_code
            if len(list_bankuai_remen) == 0:
                # print("长度为0")
                df_["板块热门"] = np.nan
                # 对热门板块进行换名
                # df_["板块热门"] = df_.板块热门.replace([""], ['NAN'])
                pass
            else:
                df_["板块热门"] = list_bankuai_remen
                # 对热门板块进行换名
                df_["板块热门"] = df_.板块热门.replace(["https://idxw.oss-cn-beijing.aliyuncs.com/icon/tag/hot_bankuai.png"], ['热门'])
            return df_
        else:
            print("爬取失败，请检查flag,msg,dxw_dict")
    """
    @dxw_gegutosave
    @geguwrapper
    def _gegu_except(day='20210528',page="1",type_page="1",position="1",new_col_=new_col_dic["col1"]):
        ct._write_console()
        # 个股涨幅榜
        if day=='20210528':
            day==""#今日
        else:
            day==day
        #个股涨幅榜
        url ='http://api.quchaogu.com/dxwapp/gegu/index?day=20210528&pagecount=20&type=1&page=1&apiversion=8.9&backgroundcolor=white&vaid=&oaid=&device_id=a865166028641465'
        #资金榜'/dxwapp/gegu/index?day=20210528&pagecount=20&type=0&page=1&apiversion=8.9&backgroundcolor=white&vaid=&oaid=&device_id=a865166028641465 HTTP/1.1'
        """
        type=1
        0: {t: "资金榜", v: 0, current: 0}
        1: {t: "涨幅榜", v: 1, current: 1}
        2: {t: "涨速榜", v: 2, current: 0}
        3: {t: "换手榜", v: 3, current: 0}
        4: {t: "成交榜", v: 4, current: 0}
        """
        #______________________________X
        headers = {
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
        'Accept-Encoding': 'gzip, deflate',
        'Accept-Language': 'zh-CN,zh;q=0.9',
        'Cache-Control': 'no-cache',
        'Connection': 'keep-alive',
        'Cookie': 'web_uinfo=%7B%22uactime%22%3A1615529397%7D; vistor_uuid=72917798314629043891129',
        'Host': 'api.quchaogu.com',
        'Pragma': 'no-cache',
        'Upgrade-Insecure-Requests': '1',
        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.82 Safari/537.36'
        }
        params={
        'day': '{}'.format(day),
        'pagecount': '20',
         ' position': '{}'.format(position),
        'type': '{}'.format(type_page),
        'page': '{}'.format(page),
        'apiversion': '8.9',
        'backgroundcolor': 'white',
        'vaid': '',
        'oaid': '',
        'device_id': 'a865166028641465 HTTP/1.1'}
        """#
        day: 20210528
        pagecount: 20
        position: 2
        type: 1
        page: 1
        orderValue: desc
        orderKey: p_day3
        apiversion: 8.9
        backgroundcolor: white
        vaid: 
        oaid: 
        device_id: a865166028641465 HTTP/1.1
        #___________________X
        0: {text: "股票", page_id: 1023}
        1: {text: "涨幅", sort: 1, sort_key: "percent"}
        2: {text: "3日涨幅", sort: 1, sort_key: "p_day3", sort_val: 1}
        3: {text: "10日涨幅", sort: 1, sort_key: "p_day10"}
        4: {text: "板块", sort_key: "bk_name"}
        5: {text: "现价", sort: 1, sort_key: "price"}
        6: {text: "股性", sort: 1, sort_key: "guxing"}
        7: {text: "主动净买", sort: 1, sort_key: "je"}
        8: {text: "主力资金", sort: 1, sort_key: "zlje"}
        9: {text: "主动占比", sort: 1, sort_key: "zj_rate"}
        10: {text: "主力占比", sort: 1, sort_key: "zl_rate"}
        11: {text: "涨速", sort: 1, sort_key: "price_speed"}
        12: {text: "换手", sort: 1, sort_key: "turnover"}
        13: {text: "量比", sort: 1, sort_key: "lb"}
        14: {text: "成交额", sort: 1, sort_key: "amount"}
        15: {text: "流通值", sort: 1, sort_key: "circulated_value"}
        16: {text: "市盈(动)", sort: 1, sort_key: "syd"}
        """
        r = requests.get(url,headers=headers,params=params)
        if r.status_code==200:
            r=r.text.encode('utf-8').decode('unicode_escape')
            if simplejson.loads(r, strict=False)["code"]==10000:#成功获取10000
                ##print(simplejson.loads(r, strict=False)["data"]["day"])#时间
                r_list=simplejson.loads(r, strict=False)["data"]["stock_list"]#stock_list

                df_ = pd.DataFrame()
                for keys,values in pd.DataFrame(r_list['list']).items():
                    text_ = pd.io.json.json_normalize(values)  # code
                    text_ = pd.DataFrame.from_records(values,columns = ["text"])
                    df_ = pd.concat([df_, text_], axis=1)

                    if keys==0:#机构与游资
                        # print("第0行",keys)
                        #print(pd.io.json.json_normalize(values)["icon_target.type"])#
                        #print(len(pd.DataFrame(r_list['list'])))#个数
                        list_=[]
                        for i in range(len(pd.DataFrame(r_list['list']))):
                            try:
                                type_num=pd.DataFrame(r_list['list'])[0][i]["icon_target"]["param"]["type"]#返回3为游资，返回6为机构
                                if type_num==6:
                                    #print("机构")
                                    list_.append("机构")
                                if type_num==3:
                                    #print("游资")
                                    list_.append("游资")

                            except:
                                #print("NAN")
                                list_.append("NAN")
                    if keys==4:#板块
                        #print("第4行",keys)
                        ##print(pd.io.json.json_normalize(values)["param.code"])#板块code
                        ##print(pd.io.json.json_normalize(values)["attr_tag"])#板块热门标志
                        list_bankuai_code=pd.io.json.json_normalize(values)["param.code"]
                        try:
                            list_bankuai_remen=pd.io.json.json_normalize(values)["attr_tag"]
                        except:
                            list_bankuai_remen=[]
                #换列名字
                new_col = new_col_#['股票', '涨幅',"3日涨幅","10日涨幅","板块","现价","股性","主动净买","主力资金","主动占比","主力占比","涨速","换手","量比","成交额","流通值","市盈(动)"]
                df_.columns = new_col

                #df_["机构"]=list_对下面进行替换/2021.9.28
                if len(df_) == len(list_):
                    df_["机构"] = list_
                else:
                    df_["机构"] ="NAN"

                df_["板块code"]=list_bankuai_code
                if len(list_bankuai_remen)==0:
                    #print("长度为0")
                    df_["板块热门"]=""
                    #对热门板块进行换名
                    # df_["板块热门"]=df_.板块热门.replace([""],['NAN'])
                    pass
                else:
                    df_["板块热门"]=list_bankuai_remen
                    #对热门板块进行换名
                    df_["板块热门"]=df_.板块热门.replace(["https://idxw.oss-cn-beijing.aliyuncs.com/icon/tag/hot_bankuai.png"],['热门'])
            return df_
        else:
            print("爬取失败，请查找源代码,错误代码()".format(r.status_code))

    def gegu(*args,**kwargs):
        try:
            # print("主方案1开始工作，gegu_except")
            res=dxw._gegu_except(day=kwargs["day"]) #dxw._gegu_except(day="20210928")
            print(res)
        except:
            print("主方案2开始工作，gegu，-----》暂时不好用")
            res=dxw._gegu()
            print("方案2开始工作，_gegu")
            print(res)

    #————————————————————————————————————————————————————————————
    def longhu(day):#模仿_gegu_except结构，后期处理模仿df.method.gegu_method3
        url = "http://api.quchaogu.com/dxwapp/lhb/index?date={}&orderValue=1&orderKey=percent&quanzi_record=%7B%221%22%3A%221633091797%22%7D&apiversion=8.9&backgroundcolor=white&vaid=&oaid=&device_id=a865166023406484 HTTP/1.1".format(day)

        r = requests.get(url, headers="").text
        r_list = simplejson.loads(r, strict=False)['data']['stock_list']  # 20210330
        df_ = pd.DataFrame()
        for keys, values in pd.DataFrame(r_list['list']).items():

            text_ = pd.io.json.json_normalize(values)
            # print(text_)
            text_ = pd.DataFrame.from_records(values, columns=["text"])
            df_ = pd.concat([df_, text_], axis=1)
            if keys == 0:
                list_ = []
                for i in range(len(pd.DataFrame(r_list['list']))):
                    list_.append(pd.DataFrame(r_list['list'])[0][i]['remark'])
            if keys == 4:
                bk_code = pd.io.json.json_normalize(values)["param.code"]

        df_['标注'] = list_
        df_['bk_code'] = bk_code
        new_col = ['股票', '涨幅', "净买入", "成交额", "板块", "上榜买比", "上榜卖比", "流通市值", "席位分析", '标注', 'bk_code']
        df_.columns = new_col
        df_['code'] = df_['股票'].str.split('#', expand=True)[1].tolist()
        df_['股票'] = df_['股票'].str.split('#', expand=True)
        df_['标注'] = df_['标注'].str.split('>', expand=True)[1].str.split('</', expand=True)
        # print(df_)
        dicTojson.dxw_path_save(df_, i_day=day, path="longhu")  # 保存
        print("->成功保存数据dxw/longhu/{}.json".format(day))
        return df_
    #————————————————————————————————————————————————————————————
    """def zhangting():
        today_time = com.time_saveandget()
        url = '{}'.format(t_u.dxw["涨停"])
        flag, msg, dxw_dic = Request_url.get(url, headers="")
        #print(flag, msg, dxw_dic)
        df = dicTojson.zhangting.main(dxw_dic)  # 变成df
        dicTojson.dxw_path_save(df, i_day=today_time, path="zhangting")  # 保存
        # json=dicTojson.dxw_jiexi(dxw_dic)#解析
        # #变美丽
        # df_list = []
        # df = pd.DataFrame()
        # for i in range(len(json)):
        #     df_ = pd.DataFrame(json[i])
        #     df_text = df_.text[:16].values.tolist()
        #     df_text = df_text + df_[:1]["remark"].values.tolist() + df_[:1]["param"].values.tolist()#增加2列
        #     df_4_5 = df_[4:5]["param"]#增加板块
        #     df_text.append(df_4_5)
        #     new = pd.DataFrame([df_text])
        #     df=pd.concat([df,new])
        # df = df.rename(
        #     columns={0: '股票', 1: '现价', 2: '涨停时刻', 3: '几天几板', 4: '板块(涨停数)', 5: '封单(元)', 6: '换手率', 7: '量比', 8: '主力资金',
        #              9: '股性', 10: '开板次数', 11: '板上成交额', 12: '成交额', 13: '流通值', 14: '总市值', 15: '昨涨幅', 16: '揭秘', 17: 'code',
        #              18: '行业'})
        # print(df)"""
    # ————————————————————————————————————————————————————————————
    def yidong():
        time_list = com.time_saveandget(choose="day_list", day=50)
        dic_ = {}
        for i_day in time_list:
            url = '{}{}'.format(t_u.dxw["异动"]["url"], i_day)
            flag, msg, dxw_dic = Request_url.get(url, headers="")
            # print(flag, msg, dxw_dic)
            dic_["{}".format(i_day)] = dicTojson.dxw_dictolist(dxw_dic)  # 加入到字典
        dicTojson.dxw_yidong_save(dic_, i_day, 'yidong')
    # ————————————————————————————————————————————————————————————
    @geguwrapper
    def gn_bkcode(*args,**kwargs):
        ct._write_console()
        global gncode
        df=tscopy.gn_bk(code_=gncode,daytime=kwargs["day"],page=kwargs["page"])
        return df

if __name__ == '__main__':

    # x=Dfcf_rank("000001").hot_rank()
    # print(Dfcf_rank("000001","now实时").hot_rank())
    # print(Dfcf_rank("000001","his历史").hot_rank())
    #_______________________________________
    #1. choose("同花顺top100")
    #2.choose('东方财富top100')




    # dxw("涨停","龙虎榜")
    # dxw("龙虎榜")
    # dxw.daban(name="zhangting", day="20210128")
    dxw.longhu("20211008")
    import  tusharecopy as tscopy
    #tscopy.get_today_all()
    #a=tscopy.search("人气",3)#("正在查找 1.人气   2.250天均线股票   3.成交量 ")
    #a = tscopy.search("20210706价格大于250天均线且换手率大于3",300)#("正在查找 1.人气   2.250天均线股票   3.成交量 ")
    print(tscopy.search("人气概念板块", 2))
    #todo
'''    def iwencai_bug(str_):
        a1 = tscopy.search("{}成交量".format(str_), 1)
        a2 = tscopy.search("{}5日成交量".format(str_), 1)
        print(a2.iloc[:,:8])


    iwencai_bug("20210909")'''


    #-----> print(dxw._gegu_except(page="31"))
    # dxw.gegu(day="20210921")
    # dxw.daban(name="longhu",day="")
    # a=dxw.gn_bkcode(gncode="GN302050",day="20210930")
    # print(a)
    # dxw._gegu_except(day="20210903")



